import entity.News;
import org.nlpcn.commons.lang.tire.domain.Forest;
import utils.IOUtils;
import utils.SplitWord;

import java.io.IOException;
import java.util.*;

/**
 * Created by 梅晨 on 2017/8/4.
 */
public class hotWord {
    public static void main(String[] args)throws IOException{
        HashMap<String,Integer> emotionTestData = IOUtils.getFilePathClassMap("C:\\Users\\梅晨\\PycharmProjects\\weiboSpider\\newsDir\\2017-08-03",null);
        HashMap<String,Integer> map = new HashMap<String, Integer>();
        List<String> contents = new ArrayList<String>();
        List<List<String>> wordslist = new ArrayList<List<String>>();
        for(String filePath : emotionTestData.keySet()){
            String content = IOUtils.getFileContent(filePath);
            HashSet<String> wordTypes = new HashSet<String>();
            wordTypes.add("n");
//            wordTypes.add("adj");
//            wordTypes.add("v");
            SplitWord splitWord = new SplitWord(wordTypes);
            String[] ss = filePath.split("\\\\");
            String title = ss[ss.length - 1];

            contents.add(title);
            List<String> words = splitWord.getSplitWords(content);
            wordslist.add(words);

        }

        for(List<String> words : wordslist){
            List<String> lastword = filter(words,contents,50);
            for(String word : lastword){
                if(!map.containsKey(word)){
                    map.put(word,1);
                }else {
                    map.put(word,map.get(word) + 1);
                }
            }
        }


        List<Map.Entry<String,Integer>> list = new ArrayList<Map.Entry<String, Integer>>(map.entrySet());
        Collections.sort(list, new Comparator<Map.Entry<String, Integer>>() {
            public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                return o1.getValue().compareTo(o2.getValue());
            }
        });
        for(int i = 0; i < list.size(); i++){
            System.out.println(list.get(i).getKey() + " : " + list.get(i).getValue());
        }
    }


    public static List<String> filter(List<String> words, List<String> contents, int percent){
        HashMap<String,Double> wordFre = new HashMap<String, Double>();
        for(String word : words){
//            if(word.length() <= 1) continue;
            if(!wordFre.containsKey(word)){
                wordFre.put(word,1.0);
            }else{
                wordFre.put(word,wordFre.get(word) + 1.0);
            }
        }
        for(String word : wordFre.keySet()){
            Double tf = wordFre.get(word) / words.size();
            double containNum = 0.0;
            for(String content : contents){
                if(content.contains(word)){
                    containNum++;
                }
            }

            Double idf = Math.log(contents.size() / (containNum + 1));
            wordFre.put(word,tf * idf);
        }
        List<String> list = getMaxTfidf(wordFre,percent);
        List<String> filterWords = new ArrayList<String>();
        for(String word : words){
            if(list.contains(word)){
                filterWords.add(word);
            }
        }
        return filterWords;
    }

    public static List<String> getMaxTfidf(HashMap<String,Double> map,int percent){
        List<Map.Entry<String,Double>> wordsEntry = new ArrayList<Map.Entry<String, Double>>(map.entrySet());
        Collections.sort(wordsEntry, new Comparator<Map.Entry<String, Double>>() {
            public int compare(Map.Entry<String, Double> o1, Map.Entry<String, Double> o2) {
                return o1.getValue().compareTo(o2.getValue());
            }
        });
        List<Map.Entry<String,Double>> result = new ArrayList<Map.Entry<String, Double>>();
        int num = map.size() * percent / 100;
        if(wordsEntry.size() < num){
            result.addAll(wordsEntry);
        }else{
            result.addAll(wordsEntry.subList(wordsEntry.size() - num,wordsEntry.size()));
        }
        List<String> words = new ArrayList<String>();
        for(Map.Entry<String,Double> entry : result){
            words.add(entry.getKey());
        }
        return words;
    }
}
